# -*- coding: utf-8 -*-
import scrapy
from njxinfang.items import BjxinfangItem
from scrapy.http import Request
from njxinfang.settings import custom_settings_for_spider2
class XinfangSpider(scrapy.Spider):
    custom_settings = custom_settings_for_spider2
    name = 'bjxinfang'
#    allowed_domains = ['nj.fang.lianjia.com/loupan']
    start_urls = ['http://bj.fang.lianjia.com/loupan/pg{}'.format(i) for i in range(1,11)]
#递归调用parse函数，
    def start_requests(self):
        #  获取新房的所有页数，放入数组all_urls
        all_url = ['http://bj.fang.lianjia.com/loupan/pg{}'.format(i) for i in range(1,11)]
        for url in all_url:
            print(url)
            yield Request(url, callback=self.parse)

    def parse(self, response):
        houses = response.xpath("/html/body/div[4]/ul[2]//li[@class='resblock-list post_ulog_exposure_scroll has-results']")
        for house in houses:
#BjxinfangItem是从items获取的函数
            item = BjxinfangItem()
#获取房子的基本数据放入item生成器函数中
            housename = house.xpath("./div/div[1]/a/text()").extract_first()
            station = house.xpath("./div/div[2]/span[1]/text()").extract_first() + '-' + house.xpath("./div/div[2]/span[2]/text()").extract_first() + '-'  + house.xpath("./div/div[2]/a/text()").extract_first()
            condition = house.xpath("./div/div[1]/span[2]/text()").extract_first()
            try:
                unitprice = house.xpath("./div/div[6]/div[1]/span[1]/text()").extract_first() + house.xpath("./div/div[6]/div[1]/span[2]/text()").extract_first()
                if unitprice == None:
                    unitprice = "没有数据"
            except:
                unitprice = "没有数据"
            try:
                mode = house.xpath("./div/a/span/text()").extract_first()
                if mode == None:
                    mode = "没有数据"
            except:
                mode = "没有数据"
            try:
                totalprice = house.xpath("./div/div[6]/div[2]/text()").extract_first()
                if totalprice == None:
                    totalprice = "没有数据"
            except:
                totalprice = "没有数据"
#               unitprice = house.xpath("./div/div[6]/div[1]/span[1]/text()").extract_first()+'元/每平方'
            try:
                constructarea = house.xpath("./div/div[3]/span/text()").extract_first()
                if constructarea == None:
                    constructarea = "没有数据"
            except:
                constructarea = "没有数据"
            item['housename'] = housename
            item['totalprice'] = totalprice
            item['unitprice'] = unitprice
            item['constructarea'] = constructarea
#            item['starttime'] = starttime
            item['station'] = station
            item['condition'] = condition
            item['mode'] = mode
            yield item

